{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tchu/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/Users/tchu/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline \n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as mtick\n",
    "import seaborn as sns\n",
    "sns.set_color_codes()\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import xml.etree.cElementTree as ET\n",
    "from matplotlib.ticker import FuncFormatter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## plot training curves in small grid env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dir = '/Users/tchu/Documents/rl_test/signal_control_results'\n",
    "plot_dir = base_dir + '/plots'\n",
    "if not os.path.exists(plot_dir):\n",
    "    os.mkdir(plot_dir)\n",
    "color_cycle = sns.color_palette()\n",
    "COLORS = {'ma2c': color_cycle[0], 'ia2c': color_cycle[1], 'iqll': color_cycle[2], \n",
    "          'iqld': color_cycle[3], 'greedy':color_cycle[4]}\n",
    "TRAIN_STEP = 1e6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neighbor_train_reward.csv, neighbor\n",
      "local_train_reward.csv, local\n",
      "global_train_reward.csv, global\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'global'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-d0ae2ed9eb2d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     55\u001b[0m         \u001b[0mgain\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfinal\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0minit\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'%s: final: %.2f'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfinal\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m \u001b[0mplot_train_curve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-12-d0ae2ed9eb2d>\u001b[0m in \u001b[0;36mplot_train_curve\u001b[0;34m(scenario, date)\u001b[0m\n\u001b[1;32m     27\u001b[0m         \u001b[0mx_std\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrolling\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwindow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;31m#         x_hi = df.value.rolling(window).max().values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m         \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_mean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mCOLORS\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlinewidth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m         \u001b[0mymin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnanmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_mean\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mx_std\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0mymax\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnanmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_mean\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mx_std\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'global'"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x576 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "window = 40\n",
    "def plot_train_curve_old(scenario='small_grid', date='jun06'):\n",
    "    cur_dir = base_dir + ('/eval_%s/%s/train_data' % (date, scenario))\n",
    "    names = ['global', 'local', 'neighbor']\n",
    "    labels = ['Centralized A2C', 'Independent A2C', 'Multi-agent A2C']\n",
    "    if scenario == 'large_grid':\n",
    "        names = names[1:]\n",
    "        labels = labels[1:]\n",
    "    dfs = {}\n",
    "    for file in os.listdir(cur_dir):\n",
    "        name = file.split('_')[0]\n",
    "        print(file + ', ' + name)\n",
    "        if name in names:\n",
    "            df = pd.read_csv(cur_dir + '/' + file)\n",
    "            dfs[name] = df\n",
    "\n",
    "    plt.figure(figsize=(8,6))\n",
    "#     ymin = min([df.value.min() for df in dfs.values()])\n",
    "#     ymax = max([df.value.max() for df in dfs.values()])\n",
    "    ymin = []\n",
    "    ymax = []\n",
    "    xmin = min([df.step.min() for df in dfs.values()])\n",
    "    for i, name in enumerate(names):\n",
    "        df = dfs[name]\n",
    "#         plt.plot(df.Step.values, df.Value.values, color=COLORS[i], linewidth=3, label=labels[i])\n",
    "        x_mean = df.value.rolling(window).mean().values\n",
    "        x_std = df.value.rolling(window).std().values\n",
    "#         x_hi = df.value.rolling(window).max().values\n",
    "        plt.plot(df.step.values, x_mean, color=COLORS[name], linewidth=3, label=labels[i])\n",
    "        ymin.append(np.nanmin(x_mean - x_std))\n",
    "        ymax.append(np.nanmax(x_mean + x_std))\n",
    "        plt.fill_between(df.step.values, x_mean - x_std, x_mean + x_std, facecolor=COLORS[name], edgecolor='none', alpha=0.3)\n",
    "    ymin = min(ymin)\n",
    "    ymax = max(ymax)\n",
    "    plt.xlim([xmin,TRAIN_STEP])\n",
    "    plt.ylim([ymin * 1.05, ymax * 0.95])\n",
    "    plt.xticks(fontsize=15)\n",
    "    plt.yticks(fontsize=15)\n",
    "    # plt.gca().xaxis.set_major_formatter(mtick.FormatStrFormatter('%.2e'))\n",
    "    plt.xlabel('Training step', fontsize=20)\n",
    "    plt.ylabel('Averaged episode reward', fontsize=20)\n",
    "    plt.legend(loc='lower right', fontsize=20)\n",
    "    plt.tight_layout()\n",
    "    # plt.savefig(plot_dir + '/small_grid_train.png')\n",
    "    plt.savefig(plot_dir + ('/%s_train.pdf' % scenario))\n",
    "    plt.close()\n",
    "\n",
    "    # calculate performance gains\n",
    "    print('final performance wrt centralized agent:')\n",
    "    ys = {}\n",
    "    for name in names:\n",
    "        y = dfs[name].value.values\n",
    "        final = np.mean(y[-window:])\n",
    "        init = np.mean(y[:window])\n",
    "        gain = final - init\n",
    "        print('%s: final: %.2f' % (name, final))\n",
    "plot_train_curve()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iqll_reward.csv, iqll\n",
      "ma2c_reward.csv, ma2c\n",
      "ia2c_reward.csv, ia2c\n"
     ]
    }
   ],
   "source": [
    "window = 100\n",
    "def plot_train_curve(scenario='large_grid', date='oct07'):\n",
    "    cur_dir = base_dir + ('/eval_%s/%s/train_data' % (date, scenario))\n",
    "    names = ['ma2c', 'ia2c', 'iqll']\n",
    "    labels = ['MA2C', 'IA2C', 'IQL-LR']\n",
    "#     names = ['ma2c', 'ia2c', 'iqld', 'iqll']\n",
    "#     labels = ['MA2C', 'IA2C', 'IQL-DNN', 'IQL-LR']\n",
    "    dfs = {}\n",
    "    for file in os.listdir(cur_dir):\n",
    "        name = file.split('_')[0]\n",
    "        print(file + ', ' + name)\n",
    "        if (name in names) and (name != 'greedy'):\n",
    "            df = pd.read_csv(cur_dir + '/' + file)\n",
    "            dfs[name] = df[df.test_id == -1]\n",
    "\n",
    "    plt.figure(figsize=(9,6))\n",
    "    ymin = []\n",
    "    ymax = []\n",
    "    \n",
    "    for i, name in enumerate(names):\n",
    "        if name == 'greedy':\n",
    "            plt.axhline(y=-972.28, color=COLORS[name], linewidth=3, label=labels[i])\n",
    "        else:\n",
    "            df = dfs[name]\n",
    "            x_mean = df.avg_reward.rolling(window).mean().values\n",
    "            x_std = df.avg_reward.rolling(window).std().values\n",
    "            plt.plot(df.step.values, x_mean, color=COLORS[name], linewidth=3, label=labels[i])\n",
    "            ymin.append(np.nanmin(x_mean - 0.5 * x_std))\n",
    "            ymax.append(np.nanmax(x_mean + 0.5 * x_std))\n",
    "            plt.fill_between(df.step.values, x_mean - x_std, x_mean + x_std, facecolor=COLORS[name], edgecolor='none', alpha=0.1)\n",
    "    ymin = min(ymin)\n",
    "    ymax = max(ymax)\n",
    "    plt.xlim([0, TRAIN_STEP])\n",
    "    if scenario == 'large_grid':\n",
    "        plt.ylim([-1600, -400])\n",
    "    else:\n",
    "        plt.ylim([-225, -100])\n",
    "    \n",
    "    def millions(x, pos):\n",
    "        return '%1.1fM' % (x*1e-6)\n",
    "\n",
    "    formatter = FuncFormatter(millions)\n",
    "    plt.gca().xaxis.set_major_formatter(formatter)\n",
    "    plt.xticks(fontsize=15)\n",
    "    plt.yticks(fontsize=15)\n",
    "    plt.xlabel('Training step', fontsize=18)\n",
    "    plt.ylabel('Average episode reward', fontsize=18)\n",
    "    plt.legend(loc='upper left', fontsize=18)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(plot_dir + ('/%s_train.pdf' % scenario))\n",
    "    plt.close()\n",
    "\n",
    "plot_train_curve(date='sep2019')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## plot training curves in large grid env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iqll_real_train_reward.csv, iqll\n",
      "ia2c_train_reward.csv, ia2c\n",
      "ma2c_real_train_reward.csv, ma2c\n",
      "iqld_train_reward.csv, iqld\n"
     ]
    }
   ],
   "source": [
    "plot_train_curve(scenario='real_net')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## plot evaluation curves in small grid env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "episode_sec = 3600\n",
    "def fixed_agg(xs, window, agg):\n",
    "    xs = np.reshape(xs, (-1, window))\n",
    "    if agg == 'sum':\n",
    "        return np.sum(xs, axis=1)\n",
    "    elif agg == 'mean':\n",
    "        return np.mean(xs, axis=1)\n",
    "    elif agg == 'median':\n",
    "        return np.median(xs, axis=1)\n",
    "\n",
    "def varied_agg(xs, ts, window, agg):\n",
    "    t_bin = window\n",
    "    x_bins = []\n",
    "    cur_x = []\n",
    "    xs = list(xs) + [0]\n",
    "    ts = list(ts) + [episode_sec + 1]\n",
    "    i = 0\n",
    "    while i < len(xs):\n",
    "        x = xs[i]\n",
    "        t = ts[i]\n",
    "        if t <= t_bin:\n",
    "            cur_x.append(x)\n",
    "            i += 1\n",
    "        else:\n",
    "            if not len(cur_x):\n",
    "                x_bins.append(0)\n",
    "            else:\n",
    "                if agg == 'sum':\n",
    "                    x_stat = np.sum(np.array(cur_x))\n",
    "                elif agg == 'mean':\n",
    "                    x_stat = np.mean(np.array(cur_x))\n",
    "                elif agg == 'median':\n",
    "                    x_stat = np.median(np.array(cur_x))\n",
    "                x_bins.append(x_stat)\n",
    "            t_bin += window\n",
    "            cur_x = []\n",
    "    return np.array(x_bins)\n",
    "    \n",
    "def plot_series(df, name, tab, label, color, window=None, agg='sum', reward=False):\n",
    "    episodes = list(df.episode.unique())\n",
    "    num_episode = len(episodes)\n",
    "    num_time = episode_sec\n",
    "    print(label, name)\n",
    "    # always use avg over episodes\n",
    "    if tab != 'trip':\n",
    "        res = df.loc[df.episode == episodes[0], name].values\n",
    "        for episode in episodes[1:]:\n",
    "            res += df.loc[df.episode == episode, name].values\n",
    "        res = res / num_episode\n",
    "        print('mean: %.2f' % np.mean(res))\n",
    "        print('std: %.2f' % np.std(res))\n",
    "        print('min: %.2f' % np.min(res))\n",
    "        print('max: %.2f' % np.max(res))\n",
    "    else:\n",
    "        res = []\n",
    "        for episode in episodes:\n",
    "            res += list(df.loc[df.episode == episode, name].values)\n",
    "        \n",
    "        print('mean: %d' % np.mean(res))\n",
    "        print('max: %d' % np.max(res))\n",
    "        \n",
    "    if reward:\n",
    "        num_time = 720\n",
    "    if window and (agg != 'mv'):\n",
    "        num_time = num_time // window\n",
    "    x = np.zeros((num_episode, num_time))\n",
    "    for i, episode in enumerate(episodes):\n",
    "        t_col = 'arrival_sec' if  tab == 'trip' else 'time_sec' \n",
    "        cur_df = df[df.episode == episode].sort_values(t_col)\n",
    "        if window and (agg == 'mv'):\n",
    "            cur_x = cur_df[name].rolling(window, min_periods=1).mean().values\n",
    "        else:\n",
    "            cur_x = cur_df[name].values    \n",
    "        if window and (agg != 'mv'):\n",
    "            if tab == 'trip':\n",
    "                cur_x = varied_agg(cur_x, df[df.episode == episode].arrival_sec.values, window, agg)\n",
    "            else:    \n",
    "                cur_x = fixed_agg(cur_x, window, agg)\n",
    "#         print(cur_x.shape)\n",
    "        x[i] = cur_x\n",
    "    if num_episode > 1:\n",
    "        x_mean = np.mean(x, axis=0)\n",
    "        x_std = np.std(x, axis=0)\n",
    "    else:\n",
    "        x_mean = x[0]\n",
    "        x_std = np.zeros(num_time)\n",
    "    if (not window) or (agg == 'mv'):\n",
    "        t = np.arange(1, episode_sec + 1)\n",
    "        if reward:\n",
    "            t = np.arange(5, episode_sec + 1, 5)\n",
    "    else:\n",
    "        t = np.arange(window, episode_sec + 1, window)\n",
    "#     if reward:\n",
    "#         print('%s: %.2f' % (label, np.mean(x_mean)))\n",
    "    plt.plot(t, x_mean, color=color, linewidth=3, label=label)\n",
    "    if num_episode > 1:\n",
    "        x_lo = x_mean - x_std\n",
    "        if not reward:\n",
    "            x_lo = np.maximum(x_lo, 0)\n",
    "        x_hi = x_mean + x_std\n",
    "        plt.fill_between(t, x_lo, x_hi, facecolor=color, edgecolor='none', alpha=0.1)\n",
    "        return np.nanmin(x_mean - 0.5 * x_std), np.nanmax(x_mean + 0.5 * x_std)\n",
    "    else:\n",
    "        return np.nanmin(x_mean), np.nanmax(x_mean)\n",
    "    \n",
    "def plot_combined_series(dfs, agent_names, col_name, tab_name, agent_labels, y_label, fig_name,\n",
    "                         window=None, agg='sum', reward=False):\n",
    "    plt.figure(figsize=(9,6))\n",
    "    ymin = np.inf\n",
    "    ymax = -np.inf\n",
    "    for i, aname in enumerate(agent_names):\n",
    "        df = dfs[aname][tab_name]\n",
    "        y0, y1 = plot_series(df, col_name, tab_name, agent_labels[i], COLORS[aname], window=window, agg=agg,\n",
    "                             reward=reward)\n",
    "        ymin = min(ymin, y0)\n",
    "        ymax = max(ymax, y1)\n",
    "    \n",
    "    plt.xlim([0, episode_sec])\n",
    "    if (col_name == 'average_speed') and ('global' in agent_names):\n",
    "        plt.ylim([0, 6])\n",
    "    elif (col_name == 'wait_sec') and ('global' not in agent_names):\n",
    "        plt.ylim([0, 3500])\n",
    "    else:\n",
    "        plt.ylim([ymin, ymax])\n",
    "    plt.xticks(fontsize=15)\n",
    "    plt.yticks(fontsize=15)\n",
    "    plt.xlabel('Simulation time (sec)', fontsize=18)\n",
    "    plt.ylabel(y_label, fontsize=18)\n",
    "    plt.legend(loc='upper left', fontsize=18)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(plot_dir + ('/%s.pdf' % fig_name))\n",
    "    plt.close()\n",
    "    \n",
    "def sum_reward(x):\n",
    "    x = [float(i) for i in x.split(',')]\n",
    "    return np.sum(x)\n",
    "\n",
    "def plot_eval_curve(scenario='large_grid', date='dec16'):\n",
    "    cur_dir = base_dir + ('/eval_%s/%s/eva_data' % (date, scenario))\n",
    "#     names = ['ma2c', 'ia2c', 'iqll', 'greedy']\n",
    "#     labels = ['MA2C', 'IA2C', 'IQL-LR', 'Greedy']\n",
    "#     names = ['iqld', 'greedy']\n",
    "#     labels = ['IQL-DNN','Greedy']\n",
    "    names = ['ia2c', 'greedy']\n",
    "    labels = ['IA2C', 'Greedy']\n",
    "    dfs = {}\n",
    "    for file in os.listdir(cur_dir):\n",
    "        if not file.endswith('.csv'):\n",
    "            continue\n",
    "        if not file.startswith(scenario):\n",
    "            continue\n",
    "        name = file.split('_')[2]\n",
    "        measure = file.split('_')[3].split('.')[0]\n",
    "        if name in names:\n",
    "            df = pd.read_csv(cur_dir + '/' + file)\n",
    "#             if measure == 'traffic':\n",
    "#                 df['ratio_stopped_car'] = df.number_stopped_car / df.number_total_car * 100\n",
    "#             if measure == 'control':\n",
    "#                 df['global_reward'] = df.reward.apply(sum_reward)\n",
    "            if name not in dfs:\n",
    "                dfs[name] = {}\n",
    "            dfs[name][measure] = df\n",
    "    \n",
    "    # plot avg queue\n",
    "    plot_combined_series(dfs, names, 'avg_queue', 'traffic', labels,\n",
    "                         'Average queue length (veh)', scenario + '_queue', window=60, agg='mv')\n",
    "    # plot avg speed\n",
    "    plot_combined_series(dfs, names, 'avg_speed_mps', 'traffic', labels,\n",
    "                         'Average car speed (m/s)', scenario + '_speed', window=60, agg='mv')\n",
    "    # plot avg waiting time\n",
    "    plot_combined_series(dfs, names, 'avg_wait_sec', 'traffic', labels,\n",
    "                         'Average intersection delay (s/veh)', scenario + '_wait', window=60, agg='mv')\n",
    "    # plot trip completion\n",
    "    plot_combined_series(dfs, names, 'number_arrived_car', 'traffic', labels,\n",
    "                         'Trip completion rate (veh/5min)', scenario + '_tripcomp', window=300, agg='sum')\n",
    "    # plot trip time\n",
    "#     plot_combined_series(dfs, names, 'duration_sec', 'trip', labels,\n",
    "#                          'Avg trip time (sec)', scenario + '_triptime', window=60, agg='mean')\n",
    "#     plot trip waiting time\n",
    "    plot_combined_series(dfs, names, 'wait_sec', 'trip', labels,\n",
    "                         'Avg trip delay (s)', scenario + '_tripwait', window=60, agg='mean')\n",
    "    plot_combined_series(dfs, names, 'reward', 'control', labels,\n",
    "                         'Step reward', scenario + '_reward', reward=True, window=6, agg='mv')\n",
    "# plot_eval_curve()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# plot evaluation curves in large grid env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IA2C avg_queue\n",
      "mean: 1.29\n",
      "std: 0.96\n",
      "min: 0.00\n",
      "max: 2.59\n",
      "Greedy avg_queue\n",
      "mean: 0.55\n",
      "std: 0.45\n",
      "min: 0.00\n",
      "max: 1.27\n",
      "IA2C avg_speed_mps\n",
      "mean: 4.30\n",
      "std: 3.60\n",
      "min: 0.00\n",
      "max: 14.26\n",
      "Greedy avg_speed_mps\n",
      "mean: 7.25\n",
      "std: 3.25\n",
      "min: 0.00\n",
      "max: 13.98\n",
      "IA2C avg_wait_sec\n",
      "mean: 80.04\n",
      "std: 58.51\n",
      "min: 0.00\n",
      "max: 187.74\n",
      "Greedy avg_wait_sec\n",
      "mean: 32.07\n",
      "std: 35.82\n",
      "min: 0.00\n",
      "max: 192.06\n",
      "IA2C number_arrived_car\n",
      "mean: 0.48\n",
      "std: 0.30\n",
      "min: 0.00\n",
      "max: 1.60\n",
      "Greedy number_arrived_car\n",
      "mean: 0.62\n",
      "std: 0.39\n",
      "min: 0.00\n",
      "max: 2.00\n",
      "IA2C wait_sec\n",
      "mean: 243\n",
      "max: 2580\n",
      "Greedy wait_sec\n",
      "mean: 111\n",
      "max: 1926\n",
      "IA2C reward\n",
      "mean: -130.35\n",
      "std: 93.89\n",
      "min: -254.10\n",
      "max: 0.00\n",
      "Greedy reward\n",
      "mean: -56.86\n",
      "std: 45.78\n",
      "min: -129.20\n",
      "max: 0.00\n"
     ]
    }
   ],
   "source": [
    "plot_eval_curve(scenario='real_net')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "peak_flow1 = 1100\n",
    "peak_flow2 = 925\n",
    "colors = 'brgm'\n",
    "ratios1 = np.array([0.4, 0.7, 0.9, 1.0, 0.75, 0.5, 0.25]) # start from 0\n",
    "ratios2 = np.array([0.3, 0.8, 0.9, 1.0, 0.8, 0.6, 0.2])   # start from 15min\n",
    "flows1 = peak_flow1 * 0.6 * ratios1\n",
    "flows2 = peak_flow1 * ratios1\n",
    "flows3 = peak_flow2 * 0.6 * ratios2\n",
    "flows4 = peak_flow2 * ratios2\n",
    "flows = [list(flows1) + [0] * 6, list(flows2) + [0] * 6, \n",
    "         [0] * 3 + list(flows3) + [0] * 3, [0] * 3 + list(flows4) + [0] * 3]\n",
    "t = np.arange(0, 3601, 300)\n",
    "t1 = t[:8]\n",
    "t2 = t[3:12]\n",
    "ts = [t1, t1, t2, t2]\n",
    "\n",
    "plt.figure(figsize=(9,6))\n",
    "labels = ['f1', 'F1', 'f2', 'F2']\n",
    "for i in range(4):\n",
    "    if i % 2 == 0:\n",
    "        plt.step(t, flows[i], where='post', color=colors[i], linestyle=':', linewidth=3, label=labels[i])\n",
    "    else:\n",
    "        plt.step(t, flows[i], where='post', color=colors[i], linewidth=6, label=labels[i], alpha=0.5)\n",
    "plt.xticks(fontsize=15)\n",
    "plt.yticks(fontsize=15)\n",
    "plt.xlabel('Simulation time (sec)', fontsize=18)\n",
    "plt.ylabel('Flow rate (veh/hr)', fontsize=18)\n",
    "plt.legend(loc='best', fontsize=18)\n",
    "plt.xlim([0, 3600])\n",
    "plt.tight_layout()\n",
    "plt.savefig(plot_dir + ('/large_grid_flow.pdf'))\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "74.88569578307643\n"
     ]
    }
   ],
   "source": [
    "x = [406.26, 364.03, 358.48, 348.21, 334.44]\n",
    "y = [299.84, 314.63, 316.57, 317.97, 319.84]\n",
    "# x = [112.92, 124.89, 131.78, 139.17, 147.11, 150.48, 155.51]\n",
    "# y = [358.56, 371.53, 379.00, 384.55, 386.97, 385.87, 384.24]\n",
    "def get_len(x, y):\n",
    "    l = 0\n",
    "    for i in range(len(x) - 1):\n",
    "        l += np.sqrt((x[i+1] - x[i]) ** 2 + (y[i+1] - y[i]) ** 2)\n",
    "    print(l)\n",
    "get_len(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mfd_points(df, dt_sec=300):\n",
    "    outputs = []\n",
    "    accs = []\n",
    "    ts = np.arange(0, 3601, dt_sec)\n",
    "    for episode in df.episode.unique():\n",
    "        cur_df = df[df.episode == episode]\n",
    "        for i in range(len(ts) - 1):\n",
    "            cur_df1 = cur_df[(cur_df.time_sec >= ts[i]) & (cur_df.time_sec < ts[i+1])]\n",
    "            outputs.append(np.sum(cur_df1.number_arrived_car.values) * 60 / dt_sec)\n",
    "            accs.append(np.mean(cur_df1.number_total_car.values))\n",
    "    return np.array(outputs), np.array(accs)\n",
    "    \n",
    "def plot_mfd_curve(scenario='real_net', date='oct07'):\n",
    "    cur_dir = base_dir + ('/eval_%s/%s/eva_data' % (date, scenario))\n",
    "    names = ['ma2c', 'ia2c', 'greedy']\n",
    "    labels = ['MA2C', 'IA2C', 'Greedy']\n",
    "    dfs = {}\n",
    "    for file in os.listdir(cur_dir):\n",
    "        if not file.endswith('traffic.csv'):\n",
    "            continue\n",
    "        if not file.startswith(scenario):\n",
    "            continue\n",
    "        name = file.split('_')[2]\n",
    "        if name not in names:\n",
    "            continue\n",
    "        df = pd.read_csv(cur_dir + '/' + file)\n",
    "        outputs, accs = get_mfd_points(df)\n",
    "        dfs[name] = (accs, outputs)\n",
    "    \n",
    "    plt.figure(figsize=(9,6))\n",
    "    styles = 'o^s'\n",
    "    for i, name in enumerate(names):\n",
    "        plt.scatter(dfs[name][0], dfs[name][1], s=80, marker=styles[i], c=COLORS[name], edgecolors='none', label=labels[i], alpha=0.75)\n",
    "    plt.xticks(fontsize=15)\n",
    "    plt.yticks(fontsize=15)\n",
    "    plt.xlabel('Accumulation (veh)', fontsize=18)\n",
    "    plt.ylabel('Output flow (veh/min)', fontsize=18)\n",
    "    plt.legend(loc='upper right', fontsize=18)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(plot_dir + ('/real_net_mfd.pdf'))\n",
    "    plt.close()\n",
    "plot_mfd_curve()     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(base_dir + '/eval_oct07/real_net_experimental_data/eva_data/real_net_ma2c_control.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "episode\n",
       "1     -22.036111\n",
       "2     -24.144444\n",
       "3     -26.236111\n",
       "4     -23.872222\n",
       "5    -102.705556\n",
       "6     -22.630556\n",
       "7     -25.965278\n",
       "8     -24.044444\n",
       "9     -20.601389\n",
       "10    -21.365278\n",
       "Name: reward, dtype: float64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('episode').reward.agg('mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
